import os
import sys
import re
import time

########################################################
# replace mention with @usr
def normMen(relArr):
    relArrUSR = []
    for wd in relArr:
        if wd.find("#") >= 0:
            continue
        if len(wd) <= 0:
            continue
        if wd[0] == "@":
            #wd = "@usr" # op1
            continue # op2
        wd = re.sub(r"\|", "", wd) # | is used as separator later for frame
        relArrUSR.append(wd)
    return relArrUSR

def normRep(item):
    for match in re.findall(r"(((\w)+)\2{3,})", item):
        if len(match[1]) == 1:
            item = re.sub(match[0], match[1]*3, item)
        if (len(match) > 2) and (len(match[2]) == 1):
            item = re.sub(match[0], match[2]*3, item)
            #print match
    return item

def getArg(item):
    if len(item) > 0:
        return "_".join(normMen(item.split(" ")))
    else:
        return item

def getNPs(outputDir, tStr):
    npHash = {}
    filePath = outputDir + r"/Chunktext_2013-01-" + tStr
    chFile = file(filePath)
    lineIdx = 0
    while 1:
        lineStr = chFile.readline()
        if len(lineStr) <= 0:
            print str(lineIdx) + " lines are processed. End of file. " + str(time.asctime())
            break
        lineStr = lineStr[:-1]
        if lineStr[-1:] == "]":
            lineStr = lineStr[:-1] + " ]"
        lineIdx += 1

        arr = lineStr.split(" ")
        #print "**** " + str(arr)
        #wArr = [w[:w.find("/")] for w in arr if w.find("/")>=0]
        bArr = [i for i in range(len(arr)) if arr[i].find("/") < 0]
        #print bArr
        npIdx = [(bArr[2*i]+1-(2*i+1), bArr[2*i+1]-1-(2*i+1)) for i in range(len(bArr)/2)]
        #print npIdx
        #npArr = ["_".join(wArr[item[0]:item[1]+1]) for item in npIdx] 
        #print npArr

        if len(npIdx) > 0:
            npHash[tStr+str(lineIdx-1)] = npIdx
        #if lineIdx % 100000 == 0:
            #print "# tweets processed: " + str(lineIdx) + " at " + str(time.asctime()) + " npHash: " + str(len(npHash))
            #break
    print "## " + str(len(npHash)) + " tweets contain np are obtained."
    return npHash

####################################
#get relation skeletons from relation file(extracted by reverb.jar)
def getRelskl_fromPOS(filename):
    print "Processing " + filename
    posFile = file(filename)
    outputDir = os.path.split(filename)[0]
    tStr = filename[-5:-3]
    npHash = getNPs(outputDir, tStr)
    outputFile = file(outputDir + r"/relSkl_2013-01-" + tStr, "w")
    lineIdx = 0
    vHash = {}
    while 1:
        lineStr = posFile.readline()
        if len(lineStr) <= 0:
            print str(lineIdx) + " lines are processed. End of file. " + str(time.asctime())
            break
        lineStr = lineStr[:-1]
        lineIdx += 1

        #print "************************* " + str(lineIdx)
        arr = lineStr.split(" ")
        #print arr
        tagArr = [word[word.rfind("/"):] for word in arr]
        tagStr = " ".join(tagArr) + " "
        #print "TagStr: " + tagStr

        # skip tweets with /FW(foreign word)
        fwArr = [1 for tag in tagArr if tag.startswith("/FW")]
        if len(fwArr) > 0:
            #print "#tweet with FW: " + lineStr
            continue

        for word in arr:
            #print word[word.rfind("/"):]
            vHash[word[word.rfind("/"):]] = 1

        # get verbs
        verbIdx = getVerbs(tagStr, arr)
        #print verbIdx
        ## merge adjacent verb rels
        newIdx = mergeAdj(verbIdx)
        #print newIdx

        # op1: take nearest NN as argument
        #resultArr = getArgs_NN(newIdx, tagArr, arr)

        # chunk version 0
        # op2: take nearest chunked NP as argument
        #'''
        if tStr+str(lineIdx-1) in npHash:
            npArr = npHash[tStr+str(lineIdx-1)]
        else:
            npArr = []
        #'''
        # op3: take nearest chunked NP(DIY) as argument
        # chunkNew version 1/2/3
        #print npArr
        #print arr
        npArr = getNPDIY(tagArr, npArr, arr)

        resultArr = getArgs_chunk(newIdx, tagArr, arr, npArr)

        #print resultArr
        if len(resultArr) > 0:
            outputFile.write(tStr+str(lineIdx-1) + "\t" + " ".join(resultArr) + "\n")
            # wrong - sepLine version
            #for item in resultArr:
                #outputFile.write(tStr+str(lineIdx-1) + "\t" + item + "\n")
            
        if lineIdx % 100000 == 0:
            print "# tweets processed: " + str(lineIdx) + " at " + str(time.asctime())
    print "Appeared tags: ",
    print sorted(vHash.keys())
    outputFile.close()
    posFile.close()

def getArgs_NN(newIdx, tagArr, arr):
    # detect argument in left and right
    # nearest NN
    resultArr = []
    for st in sorted(newIdx.keys()):
        ed = newIdx[st]

        '''
        ## get nearest /NN tagged word
        arg1 = getlnnWord(tagArr, st, arr)
        arg2 = getrnnWord(tagArr, ed, arr)
        '''

        ## get nearest /NN tagged word[s]
        arg1 = getlnnWords(tagArr, st, arr)
        arg2 = getrnnWords(tagArr, ed, arr)

        ## get relation
        rel = getWord(arr, st, ed)
        relArr = getRel(rel, arg1, arg2)

        #print "relation: ",
        #print relArr
        '''
        if len(relArr) > 0:
            resultArr.append("|".join(relArr))
        '''
        if len(relArr) != 3:
            print "Not triple."
            continue
        resultArr.append("|".join(relArr))

    return resultArr

def getWord(arr, st, ed):
    wordArr = normMen([w[:w.rfind("/")] for w in arr[st:ed+1]])
    return "_".join(wordArr)
    
def getRel(rel, arg1, arg2):
    relArr = [arg1, rel, arg2]
    #relArr = [normRep(w.lower()) for w in relArr if len(w) > 1]
    relArr = [normRep(w.lower()) for w in relArr]
    #print relArr
    return relArr

def getlnnWords(tagArr, st, arr):
    arg1 = ""
    lidx = getlnnIdx(tagArr, st)
    if lidx != -1:
        if (lidx-1 >= 0) and (tagArr[lidx-1].startswith("/NN")):
            #arg1 = "_".join(normMen([w[:w.rfind("/")] for w in arr[lidx-1:lidx+1]]))
            arg1 = getWord(arr, lidx-1, lidx)
        else:
            #arg1 = arr[lidx] 
            #arg1 = arg1[:arg1.rfind("/")]
            arg1 = getWord(arr, lidx, lidx)
    return arg1

def getrnnWords(tagArr, ed, arr):
    arg2 = "" 
    ridx = getrnnIdx(tagArr, ed)
    if ridx != -1:
        if (ridx+1 < len(tagArr)) and (tagArr[ridx+1].startswith("/NN")):
            #arg2 = "_".join(normMen([w[:w.rfind("/")] for w in arr[ridx:ridx+2]]))
            arg2 = getWord(arr, ridx, ridx+1)
        else:
            #arg2 = arr[ridx] 
            #arg2 = arg2[:arg2.rfind("/")]
            arg2 = getWord(arr, ridx, ridx)
    return arg2

def getlnnWord(tagArr, st, arr):
    arg1 = ""
    lidx = getlnnIdx(tagArr, st)
    if lidx != -1:
        #arg1 = arr[lidx] 
        #arg1 = arg1[:arg1.rfind("/")]
        arg1 = getWord(arr, lidx, lidx)
    return arg1

def getrnnWord(tagArr, ed, arr):
    arg2 = "" 
    ridx = getrnnIdx(tagArr, ed)
    if ridx != -1:
        #arg2 = arr[ridx] 
        #arg2 = arg2[:arg2.rfind("/")]
        arg2 = getWord(arr, ridx, ridx)
    return arg2

def getlnnIdx(tagArr, st):
    lidx = st - 1
    while lidx >= 0:
        if not tagArr[lidx].startswith("/NN"):
            lidx -= 1
            continue
        return lidx
    return -1

def getrnnIdx(tagArr, ed):
    ridx = ed+1
    while ridx < len(tagArr):
        if not tagArr[ridx].startswith("/NN"):
            ridx += 1
            continue
        return ridx
    return -1

def stripArr(tagArr, item, tag):
    #strip tag(/UH) in item range of tagArr
    if item is None:
        return None
    st = item[0]
    ed = item[1]
    while tagArr[st] == tag:
        st += 1
        #print "Tag: " + tag + ": " + str(item)
        if st > ed:
            return None
    while tagArr[ed] == tag:
        ed -= 1
        #print "Tag: " + tag + ": " + str(item)
        if st > ed:
            return None
    return (st, ed)

def getNewNP(tagArr, item, arr):
    item = stripArr(tagArr, item, "/UH")
    item = stripArr(tagArr, item, "/,")
    item = stripArr(tagArr, item, "/.")

    if item is None:
        return None

    newItems = []
    st = item[0]
    ed = item[1]

    # split NPs with /, /. tag
    puncArr = [id for id in range(st, ed+1) if (tagArr[id].startswith("/,") or tagArr[id].startswith("/."))]
    if len(puncArr) > 0:
        #print "NP with punc: " + "_".join(arr[item[0]:item[1]+1])
        #if len(puncArr) > 1:
            #print "##Multiple puncs"
        i = 0
        while i < len(puncArr):
            id = puncArr[i]
            if tagArr[id-1].startswith("/NN"):
                if i == 0:
                    newItems.append((st, id-1))
                else:
                    newItems.append((puncArr[i-1]+1, id-1))
            if i == len(puncArr)-1:
                if tagArr[ed].startswith("/NN"):
                    newItems.append((puncArr[i]+1, ed))
            i += 1
    else:
        newItems.append((st, ed))

    #print newItems
    '''
    # chunkNewV1-part2
    # chunkNewV2 skip this part compared with V1
    # do not use this part in chunkNewV2 and later chunkNew version(appeared in V4)
    # lead to worse precision
    items = []
    for it in newItems:
        st = it[0]
        ed = it[1]
        # delete chunk without /NN
        nounArr = [id for id in range(st, ed+1) if tagArr[id].startswith("/NN")]
        if len(nounArr) < 1:
            #print "chunk without noun: " + "_".join(arr[st: ed+1])
            continue
        usrArr = [id for id in range(st, ed+1) if not arr[id].startswith("@")]
        # check chunks with length >= 5
        if ed-st+1 >= 5:
            if ed-st+1-len(usrArr) < 5: # < 5 when delete @usr(later)
                items.append((st,ed))
            if len(nounArr) <= 3:
                items.append((st,ed))
        
    return items
    '''
    return newItems
    
def getNPDIY(tagArr, npArr, arr):
    newNPArr = []
    #print "*************************************"
    #print tagArr
    #print npArr
    for item in npArr:
        newItems = getNewNP(tagArr, item, arr)
        if newItems is not None:
            newNPArr.extend(newItems)
    #print newNPArr
    return newNPArr

def getArgs_chunk(newIdx, tagArr, arr, npArr):
    # detect argument in left and right
    if len(npArr) <= 0:
        #print "####Only verb."
        return getArgs_NN(newIdx, tagArr, arr)

    resultArr = []
    for st in sorted(newIdx.keys()):
        ed = newIdx[st]
        arg1 = "" 
        arg2 = "" 

        if len(npArr) > 0:
            idx = len(npArr)-1
            while idx >= 0:
                item = npArr[idx]
                if item[1] < st:
                    '''
                    arg1 = arr[item[0]:item[1]+1]
                    arg1 = normMen([w[:w.rfind("/")] for w in arg1])
                    arg1 = "_".join(arg1)
                    '''
                    arg1 = getWord(arr, item[0], item[1])
                    break
                idx -= 1
            idx = 0
            while idx < len(npArr):
                item = npArr[idx]
                if item[0] > ed:
                    '''
                    arg2 = arr[item[0]:item[1]+1]
                    arg2 = normMen([w[:w.rfind("/")] for w in arg2])
                    arg2 = "_".join(arg2)
                    '''
                    arg2 = getWord(arr, item[0], item[1])
                    break
                idx += 1

        #'''
        ## chunkNewV3: if no chunk is found for argument, take nearest nn as argument
        if len(arg1) <= 1:
            ## get nearest /NN tagged word
            arg1 = getlnnWord(tagArr, st, arr)
        if len(arg2) <= 1:
            arg2 = getrnnWord(tagArr, ed, arr)
        #'''

        '''
        # worse than chunkNewV3
        ## chunkNewV3_NNs: if no chunk is found for argument, take nearest NN[s] as argument
        if len(arg1) <= 1:
            ## get nearest /NN tagged word[s]
            arg1 = getlnnWords(tagArr, st, arr)
        if len(arg2) <= 1:
            arg2 = getrnnWords(tagArr, ed, arr)
        '''

        rel = getWord(arr, st, ed)
        relArr = getRel(rel, arg1, arg2)
        #print "relation: ",
        #print relArr

        #if len(relArr) > 0:
            #resultArr.append("|".join(relArr))
        if len(relArr) != 3:
            print "Not triple."
            continue
        #if len("|".join(relArr).split("|")) != 3:
            #print "Bad format: " + "|".join(relArr)
        resultArr.append("|".join(relArr))
    return resultArr

def getVerbs(tagStr, arr):
    verbIdx = {}
    i = 0
    for match in re.findall(r"(%s)"%SHORT_RELATION_PATTERN, tagStr):
        befStr = tagStr[0:tagStr.find(match[0])].strip()
        befArr = befStr.split(" ")
        if len(befStr) == 0:
            st = 0
        else:
            st = len(befArr)
        ed = st+len(match[0].strip().split(" "))-1

        # print for debug
        #print "S: match" + str(i) + ", matchedStr: ",
        #print str(match) + " ",
        #print "verbRel :" + str(arr[st:ed+1])
        #print befStr + " ",
        #print befArr

        # store into verbIdx
        if st not in verbIdx: # keep longes match
            verbIdx[st] = ed
        elif verbIdx[st] >= ed:
            continue
        elif verbIdx[st] < ed:
            verbIdx[st] = ed
        i += 1

    i = 0
    for match in re.findall(r"(%s)"%LONG_RELATION_PATTERN, tagStr):
        befStr = tagStr[0:tagStr.find(match[0])].strip()
        befArr = befStr.split(" ")
        if len(befStr) == 0:
            st = 0
        else:
            st = len(befArr)
        ed = st+len(match[0].strip().split(" "))-1

        # print for debug
        #print "L: match" + str(i) + ", matchedStr: ",
        #print str(match) + " ",
        #print "verbRel :" + str(arr[st:ed+1])

        # store into verbIdx
        if st not in verbIdx: # keep longes match
            verbIdx[st] = ed
        elif verbIdx[st] >= ed:
            continue
        elif verbIdx[st] < ed:
            verbIdx[st] = ed
        i += 1
    return verbIdx

## merge adjacent verb rels
def mergeAdj(verbIdx):
    newIdx = {}
    if len(verbIdx) == 0:
        return {}
    sts = sorted(verbIdx.keys())
    newIdx[sts[0]] = verbIdx[sts[0]]
    #print newIdx
    pst = sts[0] # previous start idx in newIdx
    for i in range(1, len(sts)):
        st = sts[i]
        if st <= newIdx[pst]:
            if newIdx[pst] < verbIdx[st]:
                newIdx[pst] = verbIdx[st]
        else:
            newIdx[st] = verbIdx[st]
            pst = st
        #print newIdx
    return newIdx

####################################
#get relation skeletons from relation file(extracted by reverb.jar)
def getRelskl_fromRel(filename):
    print "Processing " + filename
    relFile = file(filename)
    outputDir = os.path.split(filename)[0]
    tStr = filename[-2:]
    outputFile = file(outputDir + r"/relSkl_2013-01-" + tStr, "w")
    lineIdx = 0
    previousTid = tStr + "0"
    previousText = ""
    while 1:
        lineStr = relFile.readline()
        if len(lineStr) <= 0:
            print str(lineIdx) + " lines are processed. End of file. " + str(time.asctime())
            break
        lineIdx += 1
        arr = lineStr.split("\t")
        relArr = []
        #print arr

        tid = tStr+arr[1]
        arg1 = getArg(arr[-3])
        rel = "_".join(arr[-2].split(" "))
        arg2 = getArg(arr[-1][:-1])
        conf = float(arr[11])

        relArr.append(tid)
        relArr.append(normRep(arg1))
        relArr.append(normRep(rel))
        relArr.append(normRep(arg2))
        relArr.append(conf)
        print relArr
        text = "_".join(relArr[1:-1])
        if tid != previousTid:
            if len(previousText) > 1:
                outputFile.write(previousTid + "\t" + previousText + "\n")
            #print "## " + previousTid + " " + previousText
            previousTid = tid
            previousText = text
        else:
            previousText += (" "+text)
        
        if lineIdx % 100000 == 0:
            print "# tweets processed: " + str(lineIdx) + " at " + str(time.asctime())
    outputFile.close()
    relFile.close()


####################################
#main
global SHORT_RELATION_PATTERN, LONG_RELATION_PATTERN
# verb is composed of 
#// 1) Optional adverb
#// 2) Modal or other verbs
#// 3) Optional particle/adverb
VERB = r"(/RB )?" + r"(/MD |/VB |/VBD |/VBP |/VBZ |/VBG |/VBN )" + r"(/RP )?(/RB )?"
WORD = r"(/$ |/PRP$ |/CD |/DT |/JJ |/JJS |/JJR |/NN |" + r"/NNS |/NNP |/NNPS |/POS |/PRP |/RB |/RBR |/RBS |" + r"/VBN |/VBG )"
PREP = r"(/RB )?(/IN |/TO |/RP )(/RB )?"

#The pattern (V(W*P)?)+
LONG_RELATION_PATTERN = "({0} ({1}* ({2})+)?)+".format('VERB', 'WORD', 'PREP')
print "###LONG_RELATION_PATTERN: " + LONG_RELATION_PATTERN
LONG_RELATION_PATTERN = "({0}(({1})*({2})+)?)+".format(VERB, WORD, PREP)
#print LONG_RELATION_PATTERN
#The pattern (VP?)+
SHORT_RELATION_PATTERN = "({0} ({1})?)+".format('VERB', 'PREP')
print "###SHORT_RELATION_PATTERN: " + SHORT_RELATION_PATTERN
SHORT_RELATION_PATTERN = "({0}({1})?)+".format(VERB, PREP)
#print SHORT_RELATION_PATTERN

if len(sys.argv) == 2:
    filename = sys.argv[1]
    #getRelskl_fromRel(filename)
    getRelskl_fromPOS(filename)
else:
    #print "Usage getRelSkl.py relationFileName"
    print "Usage getRelSkl.py posFileName"
    sys.exit()
